378 lines (377 with data), 12.4 kB

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import roc_curve, auc, classification_report\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "from utils import print_confusion_matrix, assemble_dataset_supervised_learning\n",
    "from sklearn.ensemble import VotingClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_path = r\"./\" #path were the data is stored\n",
    "\n",
    "peaklist = np.array(pd.read_csv(r'.\\\\regions_peaklist.txt', sep = \" \")) #load the preselected of peaklist\n",
    "\n",
    "\n",
    "\n",
    "path_data = r'.\\msi_tables_filtered'\n",
    "list_dataset = os.listdir(path_data)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "labels = pd.read_csv(os.path.join(main_path,'labels_frozen.txt'),sep = ';' )\n",
    "\n",
    "full_dataset, y_labels = assemble_dataset_supervised_learning(labels,list_dataset,path_data, data_type = \"stroma\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pre-process data per patient with box cox and 10**5 factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_X_gauss = {}\n",
    "\n",
    "pt = preprocessing.PowerTransformer(method='box-cox', standardize=False)\n",
    "name_images = full_dataset[full_dataset[\"dataset_name\"]==\"SlideA1\"][\"image_name\"]\n",
    "temp_patient_data = full_dataset[full_dataset[\"dataset_name\"]==\"SlideA1\"].drop(columns = ['dataset_name','image_name'])*10**5\n",
    "X_gaus = pt.fit_transform(temp_patient_data)\n",
    "\n",
    "columns = np.unique(full_dataset[\"dataset_name\"])\n",
    "\n",
    "for col in tqdm( columns[1:]):\n",
    "    name_images = full_dataset[full_dataset[\"dataset_name\"]==col][\"image_name\"]\n",
    "    temp_patient_data = full_dataset[full_dataset[\"dataset_name\"]==col].drop(columns = ['dataset_name','image_name'])*10**5\n",
    "    array_trans = pt.fit_transform(temp_patient_data)\n",
    "    X_gaus=np.concatenate((X_gaus,array_trans),axis =0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test_and_valid, y_train, y_test_and_valid, data_train, data_test_and_valid = train_test_split(X_gaus,y_labels , full_dataset[[\"dataset_name\",'image_name']],test_size = 0.30, random_state=10) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create test dataset\n",
    "len_half = len(y_test_and_valid)//2\n",
    "X_test = X_test_and_valid[:len_half]\n",
    "data_test = data_test_and_valid[:len_half]\n",
    "y_test = y_test_and_valid[:len_half]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create validation dataset\n",
    "X_valid = X_test_and_valid[len_half:]\n",
    "data_valid = data_test_and_valid[len_half:]\n",
    "y_valid = y_test_and_valid[len_half:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##saving data train, test and valid to reuse in H&E pipeline\n",
    "\n",
    "#data_train.insert(loc=1, column='labels', value=y_train)\n",
    "#data_test.insert(loc=1, column='labels', value=y_test)\n",
    "#data_valid.insert(loc=1, column='labels', value=y_valid)\n",
    "\n",
    "#\n",
    "#data_train.to_csv('data_train_stroma_vs_epithelial_tissue.csv',index=False)\n",
    "#data_test.to_csv('data_test_stroma_vs_epithelial_tissue.csv',index=False)\n",
    "#data_valid.to_csv('data_valid_stroma_vs_epithelial_tissue.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(os.path.join(main_path,'data_train_stroma_vs_epithelial_tissue.csv'),sep = ',' )\n",
    "\n",
    "X_train = []\n",
    "y_train = []\n",
    "train_paths = []\n",
    "for slide in tqdm(os.listdir(os.path.join(main_path, 'Slides'))):\n",
    "    tile_path = os.path.join(main_path, 'Slides',slide,'tiles')\n",
    "    gland = data[(data['labels']=='stroma') & (data['dataset_name']==slide)]['image_name']\n",
    "    tissue = data[(data['labels']== \"epithelial tissue\") & (data['dataset_name']==slide)]['image_name']\n",
    "    gland = list(gland)\n",
    "    tissue = list(tissue)\n",
    "    for image_path in gland:\n",
    "        if os.path.isfile(os.path.join(tile_path, image_path)):\n",
    "            X_train.append(dict_X_gauss[slide+image_path][0])\n",
    "            y_train.append(\"stroma\")\n",
    "            train_paths.append(os.path.join(tile_path, image_path))\n",
    "        else:\n",
    "            print(\"error for gland\")\n",
    "    for image_path in tissue:\n",
    "        if os.path.isfile(os.path.join(tile_path, image_path)):\n",
    "            X_train.append(dict_X_gauss[slide+image_path][0])\n",
    "            y_train.append(\"epithelial tissue\")\n",
    "            train_paths.append(os.path.join(tile_path, image_path))\n",
    "\n",
    "y_train = np.ravel(np.array(y_train))\n",
    "            \n",
    "df_features_train = pd.DataFrame(X_train,index= train_paths,columns=list(full_dataset.columns)[2:])\n",
    "df_features_train[\"labels\"]=y_train\n",
    "#df_features_train.to_csv(r\".\\train_features_tissue_type_msi.csv\")\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(os.path.join(main_path,'data_test_stroma_vs_epithelial_tissue.csv'),sep = ',' )\n",
    "\n",
    "X_test = []\n",
    "y_test = []\n",
    "test_paths = []\n",
    "        \n",
    "for slide in tqdm(os.listdir(os.path.join(main_path, 'Slides'))):\n",
    "    tile_path = os.path.join(main_path, 'Slides',slide,'tiles')\n",
    "    gland = data[(data['labels']=='stroma') & (data['dataset_name']==slide)]['image_name']\n",
    "    tissue = data[(data['labels']=='epithelial tissue') & (data['dataset_name']==slide)]['image_name']\n",
    "    gland = list(gland)\n",
    "    tissue = list(tissue)\n",
    "    for image_path in gland:\n",
    "        if os.path.isfile(os.path.join(tile_path, image_path)):\n",
    "            X_test.append(dict_X_gauss[slide+image_path][0])\n",
    "            y_test.append(\"stroma\")\n",
    "            test_paths.append(os.path.join(tile_path, image_path))\n",
    "    for image_path in tissue:\n",
    "        if os.path.isfile(os.path.join(tile_path, image_path)):\n",
    "            X_test.append(dict_X_gauss[slide+image_path][0])\n",
    "            y_test.append(\"epithelial tissue\")\n",
    "            test_paths.append(os.path.join(tile_path, image_path))\n",
    "            \n",
    "y_test = np.ravel(np.array(y_test))\n",
    "\n",
    "df_features_test = pd.DataFrame(X_test[:len(y_test)//2],index= test_paths[:len(y_test)//2],columns=list(full_dataset.columns)[2:])\n",
    "df_features_test['labels']=y_test[:len(y_test)//2]\n",
    "#df_features_test.to_csv(r\".\\test_features_tissue_type_msi.csv\")\n",
    "\n",
    "df_features_valid = pd.DataFrame(X_test[len(y_test)//2:],index= test_paths[len(y_test)//2:],columns=list(full_dataset.columns)[2:])\n",
    "df_features_valid['labels']=y_test[len(y_test)//2:]\n",
    "#df_features_valid.to_csv(r\".\\valid_features_tissue_type_msi.csv\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gridsearchCV for MLPclassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = { 'batch_size':[32,64,128,356], 'alpha': 10.0 ** -np.arange(1, 10), 'hidden_layer_sizes':list(product(np.arange(10,21,10),np.arange(10,21,10)))}\n",
    "mlp_model = GridSearchCV(MLPClassifier(solver='adam',max_iter = 1100), parameters, n_jobs=20, , cv= 5, verbose = 2)\n",
    "\n",
    "mlp_model.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gridsearchCV for random forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = { \n",
    "   'n_estimators': [100,200,500],\n",
    "   'max_depth' : [4,8,16],\n",
    "   'criterion' :['gini', 'entropy']\n",
    "}\n",
    "\n",
    "rf_model = GridSearchCV(estimator=rfc, param_grid=param_grid, cv= 5, verbose=1,n_jobs=20)\n",
    "rf_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gridsearchCV for XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "   \"min_child_weight\":range(1,6,2),\n",
    "   \"gamma\": uniform(0, 0.5),\n",
    "   \"learning_rate\": uniform(0.03, 0.3),\n",
    "   \"max_depth\": range(3,10,2), \n",
    "   \"n_estimators\": randint(100, 150),\n",
    "   \"subsample\": uniform(0.6, 0.4)\n",
    "}\n",
    "\n",
    "xgb_model = GridSearchCV(estimator = xgb.XGBClassifier(colsample_bytree=0.8,\n",
    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27), \n",
    " param_grid = params, scoring='roc_auc',n_jobs=12,iid=False, cv=5,verbose=1)\n",
    "\n",
    "xgb_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## create feature importance file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results=pd.DataFrame()\n",
    "results['columns']=list(full_dataset.columns)[2:]\n",
    "results['importances_rf'] = CV_rfc.feature_importances_\n",
    "results['importances_xgboost'] = xgb1.feature_importances_\n",
    "results.sort_values(by='importances_xgboost',ascending=False,inplace=True)\n",
    "results.to_excel(r\".\\features_rf_xgboost_msi_gland_vs_tissue.xlsx\",index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ensemble all best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vc = VotingClassifier(estimators=[\n",
    "     ('mlp', mlp_model.best_estimator_), ('rf', rf_model.best_estimator_), ('xgb', xgb_model.best_estimator_)],\n",
    "     voting='soft',n_jobs=12)\n",
    "vc = vc.fit(np.array(X_train),np.array(y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_test =vc.predict(np.array(X_test))\n",
    "predictions_valid =vc.predict(np.array(X_valid))\n",
    "predictions_train =vc.predict(np.array(X_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## classification report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_train,predictions_train))  \n",
    "print(classification_report(y_valid,predictions_valid))  \n",
    "print(classification_report(y_test,predictions_test))  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}